#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :china_cnr_erwen.py
# @Time      :2024/3/21
# @Author    :CL
# @email     :1037654919@qq.com
# 实现对中国之声·最新播报的爬取   http://www.cnr.cn/erwen/
import datetime
import os
import shutil

import requests
from bs4 import BeautifulSoup
import json

from retrying import retry

from china_cnr_gdgg import download_m3u8_audio, merge_ts_files
from utils import mongo_manager, proxies

tts_china_cnr_erwen =  mongo_manager('tts_china_cnr_erwen',db = 'public_data')
def get_ewbfycsy(url="https://www.cnr.cn/erwen/ewbfycsy/"):
    headers = {
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Content-Type": "application/json",
        "Origin": "https://www.cnr.cn",
        "Pragma": "no-cache",
        "Referer": "https://www.cnr.cn/erwen/ewbfycsy/?id=64&lastNewsId=29108334",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-site",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    url = "https://apppc.cnr.cn/cnr45609411d2c5a16/e281277129d478c12c2ed58e84ca906b/f76a0411ae1ff31be9f9e28f0b51348b"
    data = {
        "chanId": "64",
        "pageIndex": 1,
        "perPage": 1,
        "lastNewsId": "29108334"
    }
    data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, data=data)
    response.encoding = 'gb2312'
    # print(response.text)
    print(response)
    return response.text

@retry(stop_max_attempt_number=3, wait_fixed=2000)
def get_ernew_list(
        url="https://apppc.cnr.cn/cnr45609411d2c5a16/e281277129d478c12c2ed58e84ca906b/f76a0411ae1ff31be9f9e28f0b51348b",
        data=None):
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Content-Type": "application/json",
        "Origin": "https://www.cnr.cn",
        "Pragma": "no-cache",
        "Referer": "https://www.cnr.cn/erwen/",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-site",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    if data is None:
        data = {
            "chanId": "64",
            "pageIndex": 1,
            "perPage": 60,
            "lastNewsId": "0",
            "docPubTime": ""
        }
    data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, data=data) #proxies=proxies

    response.encoding = 'utf-8'  # gbk  utf-8  gb2312
    print(response.url, response)
    return response.json()

def  get_ernew_ALL(data=None):
    page = 1
    while True:
        data = {
            "chanId": "64",  #
            "pageIndex": page,
            "perPage": 60,
            "lastNewsId": "0",
            "docPubTime": ""
        }
        res = get_ernew_list(data=data)
        # print(res['data']['categories'][0]['detail'])
        details =  res['data']['categories'][0]['detail']
        print('page:', page, 'len:', len(details))
        repeat = 0
        for detail in details:
            reslut ={}
            reslut['_id'] = detail['id']
            reslut['title'] =  detail['other_info27']
            reslut['role'] = detail['other_info13']
            reslut['duration'] = detail['other_info10']
            reslut['audio_url'] = detail['other_info9']
            reslut['img_url'] = detail['other_info24']
            reslut['img_url'] = detail['img']
            reslut['description'] = detail['description']
            reslut['time'] = detail['other_info2']
            reslut['newsContent'] = detail['newsContent']

            try:
                tts_china_cnr_erwen.insertOne(reslut)
            except Exception as e:
                repeat +=1
        if len(details) < 60:
            break
        if repeat > 50:
            break
        page += 1
def main():
    # 保存最新快报
    get_ernew_ALL()
    seeds = tts_china_cnr_erwen.findAll({'status':None})
    today  = datetime.datetime.now().strftime('%Y%m%d')
    for seed in seeds:
        audio_url = seed['audio_url']
        print(seed)
        path = f"/media/chenglei3/77D014CED257D1411/china_cnr/china.cnr.erwen_{today}/{seed['_id']}/"
        os.makedirs(path, exist_ok=True)
        if audio_url.endswith('.mp3'):  # 直接保存音频
            name = str(audio_url).split('/')[-1]
            try:
                response = requests.get(audio_url,proxies=proxies)
                if response.status_code == 200:
                    # 写入音频文件
                    with open(path + name, 'wb') as file:
                        file.write(response.content)
                    # 写入文本
                    txt_name = name.split('.')[0] + '.txt'
                    with open(path + txt_name, "w", encoding="utf-8") as file:
                        file.write(seed['newsContent'])
                    seed['status'] = 'done'
                    print(seed['_id'],seed['audio_url'],'done')
                else:
                    seed['status'] = 'error'
            except:
                seed['status'] = 'error'
        elif 'm3u8' in audio_url:
            name = str(audio_url).split('/')[-1].split('?')[0]
            # 下载.m3u8链接中的音频文件
            # 创建保存临时音频文件的文件夹
            folder_name = f"{path}{name.split('.m3u8')[0]}/"
            os.makedirs(folder_name, exist_ok=True)
            try:
                download_m3u8_audio(folder_name, audio_url)
                # 如果需要，可以将.ts文件合并为单个音频文件,并且删除folder_name下的ts文件
                merge_ts_files(input_folder=folder_name, output_file=path + name.replace('.m3u8', '.aac'))
                # 保存对应的文本
                with open(path + name.split('.m3u8')[0] + '.txt', "w", encoding="utf-8") as file:
                    file.write(seed['newsContent'])
                seed['status'] = 'done'
            except:
                seed['status'] = 'error'
        if len(os.listdir(path))<2:
            shutil.rmtree(path)
        tts_china_cnr_erwen.updateOne({'_id': seed['_id']}, seed)

if __name__ == '__main__':
    print()
    main()
    tts_china_cnr_erwen.close()
